/**
 * 
 */
package com.jjt.kiki.crawler;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.openqa.selenium.WebElement;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.JSONObject;
import com.jjt.kiki.bean.No;
import com.jjt.kiki.bean.Res;
import com.jjt.kiki.ip.IpCrawler;
import com.jjt.kiki.ip.strategy.Data5u;
import com.jjt.kiki.ip.strategy.XiciDaili;
import com.jjt.kiki.service.Frontier;
import com.jjt.kiki.service.ResService;
import com.machinepublishers.jbrowserdriver.JBrowserDriver;

/**
 * @author jjtEatJava
 *
 */
public class PanCrawler implements Runnable {
	private static final String GET_FANS = "https://pan.baidu.com/pcloud/friend/getfanslist";
	private static final String GET_FOLLOW = "https://pan.baidu.com/pcloud/friend/getfollowlist";
	private static final String GET_SHARE = "https://pan.baidu.com/pcloud/feed/getsharelist";
	private static final String SHARE_REF = "https://pan.baidu.com/share/home?uk=";
	private static final String GET_ALBUM = "https://pan.baidu.com/pcloud/album/getlist";
	private static final String QUERY_UK = "query_uk";
	private static final String LIMIT = "limit";
	private static final int LIMIT_COUNT_25 = 25;
	private static final int LIMIT_COUNT_100 = 100;
	private static final String START = "start";
	private static final ResService resService = new ResService();
	private static final String SHORTURL_PREFIX = "https://pan.baidu.com/s/";
	private static final String ALBUM_PREFIX = "https://pan.baidu.com/pcloud/album/info";
	private AtomicBoolean stopFlag = new AtomicBoolean(false);
	private Logger logger = Logger.getLogger(PanCrawler.class);

	public void stop() {
		this.stopFlag.set(true);
	}

	private void saveFans(No no) {
		if (no.getFansCount() == 0)
			return;
		String ref = SHARE_REF + no.getNoUserUk();
		Map<String, String> param = new HashMap<String, String>();
		param.put(QUERY_UK, no.getNoUserUk());
		param.put(LIMIT, LIMIT_COUNT_25+"");
		Map<String, String> header = HeaderUtil.get();
		header.put("Host", "pan.baidu.com");
		header.put("Referer", ref);
		int start = 0;
		int randomNextInt=5;
		long delay =0;
		boolean isProxy = true;

		// 轮询获取粉丝列表
		while (true) {
			param.put(START, start + "");
			String json = HttpClientPool.getInstance().create("get", GET_FANS, param, header, delay, isProxy);
			if(json==null) {
				logger.info("返回json为空，代表无法解决错误，返回");
				return;
			}
			try {
				JSONObject object = JSON.parseObject(json);
				JSONArray jsonFans = object.getJSONArray("fans_list");
				if (jsonFans.size() == 0) {
					logger.info("全部粉丝完毕");
					return;
				}
				List<No> fansList = new LinkedList<No>();
				for (int i = 0; i < jsonFans.size(); i++) {
					JSONObject fansObject = jsonFans.getJSONObject(i);
					String noUserUk = fansObject.getString("fans_uk");
					int fansCount = fansObject.getIntValue("fans_count");
					int followCount = fansObject.getIntValue("follow_count");
					int pubshareCount = fansObject.getIntValue("pubshare_count");
					int albumCount = fansObject.getIntValue("album_count");
					No fans = new No();
					fans.setFansCount(fansCount);
					fans.setFollowCount(followCount);
					fans.setNoUserUk(noUserUk);
					fans.setPubshareCount(pubshareCount);
					fans.setAlumCount(albumCount);
					fans.setNoIsFind(0);
					fansList.add(fans);
				}
				Frontier.getInstance().put(fansList);
				logger.info("一波粉丝数据生成成功");
				try {
					delay = getRandomDelay(randomNextInt);
					logger.info("HttpClient：开始休息" + delay + "豪秒");
					Thread.sleep(delay);
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
				start += LIMIT_COUNT_25;
			} catch (ParseException | JSONException e) {
				logger.info("json解析错误...页面有可能挂了...不理了继续执行");
			}
		}
	}

	private long getRandomDelay(int randomNextInt) {
		long delay;
		int random = new Random().nextInt(randomNextInt);
		delay = random *1000;
		return delay;
	}

	private void saveFollow(No no) {
		if (no.getFollowCount() == 0)
			return;
		Map<String, String> param = new HashMap<String, String>();
		param.put(QUERY_UK, no.getNoUserUk());
		param.put(LIMIT, LIMIT_COUNT_25+"");
		Map<String, String> header = HeaderUtil.get();
		String ref = SHARE_REF + no.getNoUserUk();
		header.put("Referer", ref);
		header.put("Host", "pan.baidu.com");
		int start = 0;
		int randomNextInt = 5;
		long delay = 0;
		boolean isProxy = true;
		// 轮询获取关注列表
		while (true) {
			param.put(START, start + "");
			String json = HttpClientPool.getInstance().create("get", GET_FOLLOW, param, null, delay, isProxy);
			try {
				JSONObject object = JSON.parseObject(json);
				JSONArray jsonFollow = object.getJSONArray("follow_list");
				List<No> followList = new LinkedList<No>();
				if (jsonFollow.size() == 0) {
					logger.info("全部关注获取完毕");
					return;
				}
				for (int i = 0; i < jsonFollow.size(); i++) {
					JSONObject followObject = jsonFollow.getJSONObject(i);
					String noUserUk = followObject.getString("follow_uk");
					int fansCount = followObject.getIntValue("fans_count");
					int followCount = followObject.getIntValue("follow_count");
					int pubshareCount = followObject.getIntValue("pubshare_count");
					int albumCount = followObject.getIntValue("album_count");
					No follow = new No();
					follow.setFansCount(fansCount);
					follow.setFollowCount(followCount);
					follow.setNoUserUk(noUserUk);
					follow.setPubshareCount(pubshareCount);
					follow.setAlumCount(albumCount);
					follow.setNoIsFind(0);
					followList.add(follow);
				}
				Frontier.getInstance().put(followList);
				logger.info("一波跟随数据生成成功");
			
				try {
					delay = getRandomDelay(randomNextInt);
					logger.info("HttpClient：开始休息" + delay + "豪秒");
					Thread.sleep(delay);
				} catch (InterruptedException e) {
					// 
					e.printStackTrace();
				}
				start +=LIMIT_COUNT_25;
			} catch (ParseException | JSONException e) {
				logger.info("json解析错误...页面有可能挂了...不理了继续执行");
			}
		}
	}

	/**
	 * jjtEatJava 2017年5月9日
	 * 
	 * @param no
	 */
	private void saveAlbum(No no) {
		if(no.getAlumCount()<=0)
			return;
		Map<String, String> param = new HashMap<String, String>();
		param.put(QUERY_UK, no.getNoUserUk());
		param.put(LIMIT, LIMIT_COUNT_100+"");
		Map<String, String> header = HeaderUtil.get();
		String ref = SHARE_REF + no.getNoUserUk();
		header.put("Referer", ref);
		header.put("Host", "pan.baidu.com");
		int start = 0;
		int randomNextInt = 5;
		long delay = 0;
		boolean isProxy = true;
		// 轮询获取专辑列表
		while (true) {
			param.put(START, start + "");
			String json = HttpClientPool.getInstance().create("get", GET_ALBUM, param, null, delay, isProxy);
			try {
				JSONObject object = JSON.parseObject(json);
				JSONArray albumJson = object.getJSONArray("album_list");
				List<Res> resList = new LinkedList<Res>();
				if (albumJson.size() == 0) {
					logger.info("全部专辑获取完毕");
					return;
				}
				for (int i = 0; i < albumJson.size(); i++) {
					JSONObject albumObject = albumJson.getJSONObject(i);
					String albumId = albumObject.getString("album_id");
					String title = albumObject.getString("title");
					Date date = new Date(albumObject.getLongValue("create_time"));
					Res res = new Res();
					res.setResCralwerDate(new Date());
					res.setResIsShare(0);
					res.setResShareDate(date);
					res.setResShortUrl(albumId);
					res.setResTitle(title);
					res.setResUrl(createAlbumUrl(no.getNoUserUk(),albumId));
					res.setResUserUk(no.getNoUserUk());
					resList.add(res);
				}
				int count =  resService.save(resList);
				logger.info("存入非重复专辑数："+count);
				if(count==0) {
					logger.info("访问极限，无法突破");
					return;
				}
					
				logger.info("一波专辑生成功");
			
				try {
					delay = getRandomDelay(randomNextInt);
					logger.info("HttpClient：开始休息" + delay + "豪秒");
					Thread.sleep(delay);
				} catch (InterruptedException e) {
					// 
					e.printStackTrace();
				}
				start += LIMIT_COUNT_100;
			} catch (ParseException | JSONException e) {
				logger.info("json解析错误...页面有可能挂了...不理了继续执行");
			}
		}
	}

	//用于创建专辑链接
	private String createAlbumUrl(String noUserUk, String albumId) {
		return ALBUM_PREFIX+"?uk="+noUserUk+"&"+"album_id="+albumId;
	}

	/**
	 * 使用模拟浏览器的方式请求 private void savePubshare(No no) { if (no.getAlumCount() ==
	 * 0) return; String url = "https://pan.baidu.com/share/home?uk="; url = url
	 * + no.getNoUserUk(); logger.info("执行第一次请求"); int page = 1; JBrowserDriver
	 * driver = new JBrowserDriver(); driver.get(url); driver.pageWait();
	 * List<Res> resList = analyze(driver.getPageSource(),no.getNoUserUk()); int
	 * count = resService.save(resList); logger.info("当前页面数:"+page);
	 * logger.info("第一次存入非重复资源数：" + count); // 获取下一页按钮 WebElement next =
	 * driver.findElementByClassName("page-next mou-evt"); if (next == null) {
	 * logger.info("当前页为最后一页"); return; } else { while
	 * (!next.getAttribute("class").matches(".*disabled.*")) {
	 * logger.info("还有下一页"); int random = new Random().nextInt(20);
	 * if(random<10) random= random+10; logger.info("开始休眠："+random+"秒"); try {
	 * Thread.sleep(random*1000); } catch (InterruptedException e) { // 
	 * 自动生成的 catch 块 e.printStackTrace(); } next.click(); driver.pageWait();
	 * page++; List<Res> resList2 =
	 * analyze(driver.getPageSource(),no.getNoUserUk()); int count2 =
	 * resService.save(resList2); logger.info("当前页面数:"+page);
	 * logger.info("非一次存入非重复资源数：" + count2); next =
	 * driver.findElementByClassName("page-next mou-evt"); }
	 * logger.info("没有下一页了..."); return; } }
	 * 
	 * private List<Res> analyze(String pageSource,String uk) { List<Res> list
	 * =new ArrayList<Res>(); Document doc = Jsoup.parse(pageSource); //获取资源列表
	 * Element resE = doc.getElementById("infiniteListView"); List<Element>
	 * resList = resE.children(); for(Element e : resList) { Res res = new
	 * Res(); String link = e.attr("_link"); res.setResCralwerDate(new Date());
	 * try { res.setResShareDate(new SimpleDateFormat("yyyy-MM-dd
	 * HH:mm").parse(e.child(1).text())); } catch (java.text.ParseException e1)
	 * { // 
	 * res.setResShortUrl(link.substring(link.lastIndexOf("/")+1,link.length()));
	 * res.setResTitle(e.child(0).attr("title")); res.setResUrl(link);
	 * res.setResUserUk(uk); list.add(res); } return list; }
	 **/
	/**
	 * jjtEatJava 2017年5月9日
	 * 
	 * @param no
	 */
	private void savePubshare(No no) {
		if (no.getPubshareCount() == 0)
			return;
		String ref = SHARE_REF + no.getNoUserUk();
		Map<String, String> header = HeaderUtil.get();
		header.put("Referer", ref);
		header.put("Host", "pan.baidu.com");
		Map<String, String> param = new HashMap<String, String>();
		param.put(QUERY_UK, no.getNoUserUk());
		param.put(LIMIT, LIMIT_COUNT_100+"");
		param.put("auth_type", "1");
		param.put("category", "0");
		param.put("request_location", "share_home");
		param.put("channel", "chunlei");
		param.put("clienttype", "0");
		param.put("web", "1");
		param.put("web", "1");
		param.put("t", "1495001525186");
		param.put("logid", "MTQ5NTAwMTUyNTE5OTAuNzM5ODg3OTU0NTE4MzgzNg");
		int start = 0;
		int randomNextInt =10;
		long delay = 0;
		boolean isProxy = true;
		while (true) {
			param.put(START, start + "");
			String json = HttpClientPool.getInstance().create("get", GET_SHARE, param, header, delay, isProxy);
			File file = new File("D:\\test.txt");
			if (!file.exists())
				try {
					file.createNewFile();
				} catch (IOException e2) {
					e2.printStackTrace();
				}

			try {
				FileWriter writer = new FileWriter(file, true);
				writer.write("start:" + start + " uk:" + no.getNoUserUk());
				writer.write("\n");
				writer.write(json);
				writer.write("\n");
				writer.write("---------------------");
				writer.write("\n");
				writer.close();
			} catch (IOException e1) {
				e1.printStackTrace();
			}

			try {
				JSONObject object = JSON.parseObject(json);
				Integer errno = object.getInteger("errno");
				if (errno != 0) {// 表示错误
					logger.info("请求分享错误 错误代码：" + errno);
					continue;
				}
				Integer totalCount = object.getInteger("total_count");
				if (totalCount == 0) {
					logger.info("全部分享获取完毕");
					return;
				}
				List<Res> resList = new LinkedList<Res>();
				JSONArray records = object.getJSONArray("records");
				if (records.size() == 0) {
					logger.info("全部分享获取完毕");
					return;
				}
				for (int i = 0; i < records.size(); i++) {
					JSONObject shareObject = records.getJSONObject(i);
					String title = shareObject.getString("title");
					String shortUrl = shareObject.getString("shorturl");
					long feedTime = shareObject.getLongValue("feed_time");
					Res res = new Res();
					res.setResCralwerDate(new Date());
					res.setResShareDate(new Date(feedTime));
					res.setResTitle(title);
					res.setResUrl(SHORTURL_PREFIX + shortUrl);
					res.setResUserUk(no.getNoUserUk());
					res.setResShortUrl(shortUrl);
					res.setResIsShare(1);
					resList.add(res);
				}
				int count = resService.save(resList);
				logger.info("存入非重复资源数：" + count);
				if (count == 0) {
					logger.info("访问极限，无法突破");
					return;
				}
			
				try {
					delay = getRandomDelay(randomNextInt);
					logger.info("HttpClient：开始休息" + delay + "豪秒");
					Thread.sleep(delay);
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
				start += LIMIT_COUNT_100;
			} catch (ParseException | JSONException e) {
				e.printStackTrace();
				logger.info("json解析错误...页面有可能挂了...不理了继续执行");
			}
		}
	}

	/*
	 * （非 Javadoc）
	 * 
	 * @see java.lang.Runnable#run()
	 */
	@Override
	public void run() {
		while (true) {
			if (this.stopFlag.get()) {
				logger.info("程序结束");
				return;
			}
			// 获取一个种子ip
			No no = Frontier.getInstance().take();
			if (no == null) {
				logger.info("数据库记录为空...结束程序");
				return;
			}
//			 this.saveFans(no);
//			 this.saveFollow(no);
			this.savePubshare(no);
			this.saveAlbum(no);
			Frontier.getInstance().done(no);
		}
	}

	public static void main(String[] args) {
		PanCrawler panCrawler1 = new PanCrawler();
		PanCrawler panCrawler2 = new PanCrawler();
		IpCrawler ipCrawler1 = new IpCrawler(new Data5u(), 60* 1000);
		IpCrawler ipCrawler2 = new IpCrawler(new XiciDaili(), 60* 1000);
		//新建一个线程池，里面执行四条线程，主要执行两个任务，1代理IP爬取、2百度网盘爬取
		ExecutorService executor = Executors.newFixedThreadPool(4);
		//执行代理ip爬虫
		executor.execute(ipCrawler2);
		executor.execute(ipCrawler1);
		
		//执行百度网盘爬虫
		executor.execute(panCrawler1);
		executor.execute(panCrawler2);
		
		try {
			Thread.sleep(6 *60* 10 * 1000);
			panCrawler1.stop();
			panCrawler2.stop();
			ipCrawler2.stop();
			ipCrawler1.stop();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
}
